from urllib.error import HTTPError, URLError
import logging
import requests
import csv
from scrapy.http import TextResponse
from collections import deque
from bs4 import BeautifulSoup
from bs4 import NavigableString
from urllib.parse import urlparse
from urllib import parse
import urllib
import time
import datetime
import os
import json


def create_logger():
    from logging.handlers import RotatingFileHandler

    logpath = os.path.normpath('logs/py_log.log')
    logdir = os.path.dirname(logpath)
    if not os.path.exists(logdir):
        os.makedirs(logdir)
    logging.basicConfig(
        handlers=[RotatingFileHandler(filename=os.path.normpath('logs/py_log.log'),
                                      mode='a', maxBytes=5000000, backupCount=10)],
        level=logging.DEBUG,
        format="[%(asctime)s] %(levelname)s [%(name)s.%(funcName)s:%(lineno)d] %(message)s",
        datefmt='%Y-%m-%dT%H:%M:%S')

    logFormatter = logging.Formatter("%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s]  %(message)s")
    rootLogger = logging.getLogger()

    consoleHandler = logging.StreamHandler()
    consoleHandler.setFormatter(logFormatter)
    rootLogger.addHandler(consoleHandler)


create_logger()

host = 'http://www.a-hospital.com'

#http://www.pharmnet.com.cn/tcm/yssl/
#https://www.360kad.com/Medication/mxby/
#https://ask.360kad.com/c-130020.html
#http://jf.360kad.com/ysjf/
#http://jf.360kad.com/jfcp/
#http://jf.360kad.com/ydjf/pbjf/
#http://jf.360kad.com/ydjf/ygjf/
#https://ask.360kad.com/c-193.html
#http://sex.360kad.com/xsh/fqxsh/
#https://ask.360kad.com/m-150321.html
#https://ask.360kad.com/j-152181.html
#http://pfk.360kad.com/pfmr/
#https://www.360kad.com/Medication/SkinMedication/


source_dict = {}

res = requests.get(host)

response = TextResponse(res.url, body=res.text, encoding='utf-8')

soup = BeautifulSoup(response.text, 'html.parser')



logging.info("_____________________________________________________")
logging.info("_____________________________________________________")
logging.info("test completed")
